

#include <stdio.h>
#include <tchar.h>
#include "TempleteFea.h"
#include "utilfun.h"
#include "utilio.h"
#include <iostream>
#include "binstore.h"

int _tmain(int argc, _TCHAR* argv[])
{
	
	char* mode_check = "construct";
	int i;
	if ((i = ArgPos((char *)"-mode", argc, argv)) > 0) mode_check = (argv[i + 1]);
	if(strcmp(mode_check,"construct")) {
		printf("Extract features mode error, exit!\n");
		return 0;
	}
	///////////////////////////////////////////////////////////////////////
	char* trainfile = "TongyiciCilin_train.txt";
	char* intmodepath = "fea_idxwordmd";
	char* templetepath = "template";
	char* sentence_path = "./sentence/sen_pre";
	int	  numskip_check = 20;
	int	  num_parts = 1;
	int   id_part = 0;
	int   start_save_count = 0;
	char* order="first";
	if ((i = ArgPos((char *)"-fea_wordmd", argc, argv)) > 0) intmodepath = (argv[i + 1]);
	if ((i = ArgPos((char *)"-template", argc, argv)) > 0) templetepath = (argv[i + 1]);
	if ((i = ArgPos((char *)"-trainfile", argc, argv)) > 0) trainfile = (argv[i + 1]);
	if ((i = ArgPos((char *)"-skipcheck", argc, argv)) > 0) numskip_check = atoi(argv[i + 1]);
	if ((i = ArgPos((char *)"-parts", argc, argv)) > 0) num_parts = atoi(argv[i + 1]);
	if ((i = ArgPos((char *)"-ipart", argc, argv)) > 0) id_part = atoi(argv[i + 1]);
	if ((i = ArgPos((char *)"-order", argc, argv)) > 0) order = (argv[i + 1]);
	if ((i = ArgPos((char *)"-outsen", argc, argv)) > 0) sentence_path = (argv[i + 1]);
	if ((i = ArgPos((char *)"-start_saveid", argc, argv)) > 0) start_save_count = atoi(argv[i + 1]);
	if ((i = ArgPos((char *)"-cliptag", argc, argv)) > 0)
	{
		int cliptag = atoi(argv[i + 1]);
		SetTagClip(cliptag);
	}
	bool bfirst = !strcmp(order,"first");
	if(bfirst)
		printf("Running first order tagging!\n");
	else
		printf("Running zero order tagging!\n");

	vector<sen_table> corpors;
	ReadCiLin(trainfile,corpors);
	printf("ReadCiLin finished!\n");


	TempleteFea* fea = new TempleteFea(1,10);
	fea->load_model(intmodepath,true);
	printf("Load model finished!\n");

	memPool<char> vec_pools;
	vector<sentence_inter> sen_pool;
	vector<sentence> sen_poolck;
	int isave_file_count =0;

	int begin_id=0,end_id = corpors.size();
	float part_count = corpors.size()/(float)num_parts;
	begin_id = id_part*part_count;
	end_id = id_part*part_count+part_count;
	isave_file_count = floor( begin_id/31.0f )+id_part;

	if(num_parts==1 && id_part==0) {
		begin_id = start_save_count*31;
		isave_file_count = start_save_count;
	}
	printf("[%d,%d)-->%d",begin_id,end_id,isave_file_count);
	for(int i=begin_id;i<end_id && i<corpors.size();i++)
	{
		sentence_rawin rawin;
		InitSenRawin(corpors[i],rawin);
		sentence_inter vf;
		if(bfirst)
			fea->generate_feature(rawin,vf);
		else
			fea->generate_feature_zero(rawin,vf);
		sen_pool.push_back(vf);
		printf("...done %d..\n",i);

		if(sen_pool.size()>30) {
			char filebuf[512];
			sprintf(filebuf,"%s[%d]",sentence_path,isave_file_count++);
			printf("writing %s--%f\n",filebuf,i/(float)corpors.size());

			FILE* fout = fopen(filebuf,"wb");
			write_senbin(fout,sen_pool);
			fclose(fout);
			if( !(isave_file_count % numskip_check) )
			{
				read_senbin(filebuf,sen_poolck,vec_pools);
				//------------------------------------------
				assert_loc(sen_poolck.size()==sen_pool.size());
				for(int is=0;is<sen_poolck.size();is++)
				{
					sentence_inter& sit = sen_pool[is];
					sentence& st = sen_poolck[is];
					assert_loc(sit.l==st.l);
					for(int iw=0;iw<sit.l;iw++)
					{
						assert_loc(sit.label[iw]==st.labels[iw]);
						assert_loc(sit.word_feas[iw].size()==st.word_feas_num[iw]);
						for(int ispn=0;ispn<st.word_feas_num[iw];ispn++)
						{
							span_feature& sp1 = sit.word_feas[iw][ispn];
							span_feature& sp2 = st.word_feas[iw][ispn];
							assert_loc(sp1.y_pre==sp2.y_pre);
							assert_loc(sp1.y_cur==sp2.y_cur);
							assert_loc(sp1.index==sp2.index);
							assert_loc(sp1.tpl  ==sp2.tpl);
						}
					}
				}
				printf("check %s ok\n",filebuf);
			}
			//------------------------------------------
			sen_poolck.clear();
			sen_pool.clear();
		}

	}
	return 0;
}
